package com.ady.collect;

import com.ady.util.FrameUtil;
import com.ady.util.IDUtil;
import com.ady.util.StringUtil;
import com.ady.webmgic.util.WebMagicUtil;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.experimental.Accessors;
import org.apache.commons.io.FileUtils;
import org.springframework.util.StringUtils;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;

import java.io.File;
import java.io.IOException;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;

/**
 * <b>description</b>：采集dytt <br>
 * <b>time</b>：2018-06-22 11:14 <br>
 * <b>author</b>： ready likun_557@163.com
 */
public class DyttCollect implements PageProcessor {

    public Map<String, FilmModel> dataMap1 = Collections.synchronizedMap(new LinkedHashMap<>());
    public Map<String, FilmModel> dataMap = Collections.synchronizedMap(new LinkedHashMap<>());

    @Override
    public void process(Page page) {
        Object page1 = page.getRequest().getExtra("page");
        if (page1.equals("1")) {
            for (String s : page.getHtml().css("a.ulink", "abs:href").all()) {
                Request request = new Request();
                request.setUrl(s);
                request.putExtra("page", "2");
                page.addTargetRequest(request);
            }
        } else if (page1.equals("2")) {
            String title = page.getHtml().css("div.title_all font", "text").get();
            String ftpUrl = page.getHtml().css("a[href^=ftp]", "abs:href").get();
            if (!StringUtils.isEmpty(title) && !StringUtils.isEmpty(ftpUrl)) {
                title = title.trim();
                ftpUrl = ftpUrl.trim();
                FilmModel filmModel = FilmModel.builder().id(IDUtil.getId()).title(title).ftpUrl(ftpUrl).build().initExtMap();
                this.dataMap.put(title, filmModel);
                this.dataMap1.put(title, filmModel);
            }
        }

    }

    static Map<String, String> defaultHeaderMap = FrameUtil.newHashMap(
            "Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
            "Accept-Encoding", "gzip, deflate",
            "Accept-Language", "zh-CN,zh;q=0.9",
            "Cache-Control", "max-age=0",
            "Connection", "keep-alive",
            "Host", "www.dytt8.net",
            "If-Modified-Since", "Mon, 18 Jun 2018 05:04:39 GMT",
            "If-None-Match", "\"808dffdbc16d41:31c\"",
            "Referer", "http://www.dytt8.net/html/gndy/dyzz/list_23_1.html",
            "Upgrade-Insecure-Requests", "1",
            "User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36");

    @Override
    public Site getSite() {
        return WebMagicUtil.me().setCycleRetryTimes(3).setSleepTime(100);
    }

    public static void main(String[] args) throws IOException {
        for (int i = 1; i <= 1; i++) {
            try {
                DyttCollect pageProcessor = new DyttCollect();
                Request request = new Request();
//                request.getHeaders().putAll(defaultHeaderMap);
                request.putExtra("page", "1");
                request.setUrl(String.format("http://www.dytt8.net/html/gndy/dyzz/list_23_%s.html", i));
                WebMagicUtil.create(pageProcessor).addRequest(request).thread(1).run();
                if (!pageProcessor.dataMap1.isEmpty()) {
                    String json = FrameUtil.json(pageProcessor.dataMap, true);
                    FileUtils.writeStringToFile(new File("E:\\ady\\doc\\film\\dytt.json"), json);
                    String json1 = FrameUtil.json(pageProcessor.dataMap1, true);
                    FileUtils.writeStringToFile(new File(String.format("E:\\ady\\doc\\film\\dytt_%s.json", i)), json1);
                    pageProcessor.dataMap1.clear();
                }
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
    }
}
